#!/usr/bin/env python
# encoding: utf-8
"""
Author: nghuyong
Mail: nghuyong@163.com
Created Time: 2019-12-07 21:27
"""
from __future__ import absolute_import

import os
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
from spiders.tweet_by_user_id import TweetSpiderByUserID
from spiders.tweet_by_keyword import TweetSpiderByKeyword
from spiders.tweet_by_tweet_id import TweetSpiderByTweetID
# from spiders import CommentSpider
from spiders.follower import FollowerSpider
from spiders.user import UserSpider
from spiders.fan import FanSpider
from spiders.repost import RepostSpider
import datetime

def aaa():
    get_project_settings()
if __name__ == '__main__':
    mode = 'tweet_by_keyword'
    os.environ['SCRAPY_SETTINGS_MODULE'] = 'settings'
    current_path = os.getcwd()
    # aaa()
    settings = get_project_settings()
    settings_dict = dict(settings)
    settings_dict['MONGO_DATABASE'] = 'name_new'
    process = CrawlerProcess(settings_dict)
    mode_to_spider = {
        # 'comment': CommentSpider,
        'fan': FanSpider,
        'follow': FollowerSpider,
        'user': UserSpider,
        'repost': RepostSpider,
        'tweet_by_tweet_id': TweetSpiderByTweetID,
        'tweet_by_user_id': TweetSpiderByUserID,
        'tweet_by_keyword': TweetSpiderByKeyword,
    }
    start_time = '2024-03-03 10'
    end_time = '2024-04-02 11'
    process.crawl(mode_to_spider[mode],keyword=['核废水'], start_time=start_time, end_time=end_time, is_split_by_hour=0, max_count=10)
    # the script will block here until the crawling is finished
    process.start()
